


package LXF_code.app_L.DIM;

import LXF_code.app_L.BaseApp_1;
import LXF_code.util_L.kafkaSourceUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.atguigu.realtime.common.Constant;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStreamSink;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;

public class DimApp_L extends BaseApp_1 {
    public static void main(String[] args) {

        //设置后端
        System.setProperty("HADOOP_USER_NAME", "atguigu");
        Configuration conf = new Configuration();
        conf.setInteger("rest.port", 2000);
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf);
        env.setParallelism(2);

        env.enableCheckpointing(3000);
        env.setStateBackend(new HashMapStateBackend());
        env.getCheckpointConfig().setCheckpointStorage("hdfs://hadoop162:8020/gmall/DimApp" );
        env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
        env.getCheckpointConfig().setCheckpointTimeout(20 * 1000);
        env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
        env.getCheckpointConfig().setMinPauseBetweenCheckpoints(500);
        env.getCheckpointConfig().setExternalizedCheckpointCleanup(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);

        //获取主数据流
        DataStreamSource<String> KafkaStream = env.addSource(kafkaSourceUtil.getKafkafSource("DimApp", Constant.TOPIC_ODS_DB));

        //主数据流ETL
        DataStreamSink<JSONObject> ETLstream = KafkaStream
                .filter(json -> {

                    try {
                        return "gmall2022".equals(JSON.parseObject(json).getString("database"))
                                && (
                                "insert".equals(JSON.parseObject(json).getString("type"))
                                        || "update".equals(JSON.parseObject(json).getString("type"))
                                        || "bootstrap_insert".equals(JSON.parseObject(json).getString("type"))
                        )
                                && JSON.parseObject(json).getString("data") != null
                                && JSON.parseObject(json).getString("data").length() > 2;

                    } catch (Exception e) {
                        System.out.println("json 格式有误, 你的数据是: " + json);
                        return false;
                    }
                })
                .map(JSON::parseObject).print();

            //配置流


        try {
            env.execute();
        } catch (Exception e) {
            e.printStackTrace();
        }


    }


}
